{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gensim, logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-12 16:06:33,788 : INFO : loading projection weights from GoogleNews-vectors-negative300.bin\n",
      "2017-11-12 16:08:10,091 : INFO : loaded (3000000, 300) matrix from GoogleNews-vectors-negative300.bin\n"
     ]
    }
   ],
   "source": [
    "gmodel = gensim.models.KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.0123291 ,  0.20410156, -0.28515625,  0.21679688,  0.11816406,\n",
       "        0.08300781,  0.04980469, -0.00952148,  0.22070312, -0.12597656,\n",
       "        0.08056641, -0.5859375 , -0.00445557, -0.296875  , -0.01312256,\n",
       "       -0.08349609,  0.05053711,  0.15136719, -0.44921875, -0.0135498 ,\n",
       "        0.21484375, -0.14746094,  0.22460938, -0.125     , -0.09716797,\n",
       "        0.24902344, -0.2890625 ,  0.36523438,  0.41210938, -0.0859375 ,\n",
       "       -0.07861328, -0.19726562, -0.09082031, -0.14160156, -0.10253906,\n",
       "        0.13085938, -0.00346375,  0.07226562,  0.04418945,  0.34570312,\n",
       "        0.07470703, -0.11230469,  0.06738281,  0.11230469,  0.01977539,\n",
       "       -0.12353516,  0.20996094, -0.07226562, -0.02783203,  0.05541992,\n",
       "       -0.33398438,  0.08544922,  0.34375   ,  0.13964844,  0.04931641,\n",
       "       -0.13476562,  0.16308594, -0.37304688,  0.39648438,  0.10693359,\n",
       "        0.22167969,  0.21289062, -0.08984375,  0.20703125,  0.08935547,\n",
       "       -0.08251953,  0.05957031,  0.10205078, -0.19238281, -0.09082031,\n",
       "        0.4921875 ,  0.03955078, -0.07080078, -0.0019989 , -0.23046875,\n",
       "        0.25585938,  0.08984375, -0.10644531,  0.00105286, -0.05883789,\n",
       "        0.05102539, -0.0291748 ,  0.19335938, -0.14160156, -0.33398438,\n",
       "        0.08154297, -0.27539062,  0.10058594, -0.10449219, -0.12353516,\n",
       "       -0.140625  ,  0.03491211, -0.11767578, -0.1796875 , -0.21484375,\n",
       "       -0.23828125,  0.08447266, -0.07519531, -0.25976562, -0.21289062,\n",
       "       -0.22363281, -0.09716797,  0.11572266,  0.15429688,  0.07373047,\n",
       "       -0.27539062,  0.14257812, -0.0201416 ,  0.10009766, -0.19042969,\n",
       "       -0.09375   ,  0.14160156,  0.17089844,  0.3125    , -0.16699219,\n",
       "       -0.08691406, -0.05004883, -0.24902344, -0.20800781, -0.09423828,\n",
       "       -0.12255859, -0.09472656, -0.390625  , -0.06640625, -0.31640625,\n",
       "        0.10986328, -0.00156403,  0.04345703,  0.15625   , -0.18945312,\n",
       "       -0.03491211,  0.03393555, -0.14453125,  0.01611328, -0.14160156,\n",
       "       -0.02392578,  0.01501465,  0.07568359,  0.10742188,  0.12695312,\n",
       "        0.10693359, -0.01184082, -0.24023438,  0.0291748 ,  0.16210938,\n",
       "        0.19921875, -0.28125   ,  0.16699219, -0.11621094, -0.25585938,\n",
       "        0.38671875, -0.06640625, -0.4609375 , -0.06176758, -0.14453125,\n",
       "       -0.11621094,  0.05688477,  0.03588867, -0.10693359,  0.18847656,\n",
       "       -0.16699219, -0.01794434,  0.10986328, -0.12353516, -0.16308594,\n",
       "       -0.14453125,  0.12890625,  0.11523438,  0.13671875,  0.05688477,\n",
       "       -0.08105469, -0.06152344, -0.06689453,  0.27929688, -0.19628906,\n",
       "        0.07226562,  0.12304688, -0.20996094, -0.22070312,  0.21386719,\n",
       "       -0.1484375 , -0.05932617,  0.05224609,  0.06445312, -0.02636719,\n",
       "        0.13183594,  0.19433594,  0.27148438,  0.18652344,  0.140625  ,\n",
       "        0.06542969, -0.14453125,  0.05029297,  0.08837891,  0.12255859,\n",
       "        0.26757812,  0.0534668 , -0.32226562, -0.20703125,  0.18164062,\n",
       "        0.04418945, -0.22167969, -0.13769531, -0.04174805, -0.00286865,\n",
       "        0.04077148,  0.07275391, -0.08300781,  0.08398438, -0.3359375 ,\n",
       "       -0.40039062,  0.01757812, -0.18652344, -0.0480957 , -0.19140625,\n",
       "        0.10107422,  0.09277344, -0.30664062, -0.19921875, -0.0168457 ,\n",
       "        0.12207031,  0.14648438, -0.12890625, -0.23535156, -0.05371094,\n",
       "       -0.06640625,  0.06884766, -0.03637695,  0.2109375 , -0.06005859,\n",
       "        0.19335938,  0.05151367, -0.05322266,  0.02893066, -0.27539062,\n",
       "        0.08447266,  0.328125  ,  0.01818848,  0.01495361,  0.04711914,\n",
       "        0.37695312, -0.21875   , -0.03393555,  0.01116943,  0.36914062,\n",
       "        0.02160645,  0.03466797,  0.07275391,  0.16015625, -0.16503906,\n",
       "       -0.296875  ,  0.15039062, -0.29101562,  0.13964844,  0.00448608,\n",
       "        0.171875  , -0.21972656,  0.09326172, -0.19042969,  0.01599121,\n",
       "       -0.09228516,  0.15722656, -0.14160156, -0.0534668 ,  0.03613281,\n",
       "        0.23632812, -0.15136719, -0.00689697, -0.27148438, -0.07128906,\n",
       "       -0.16503906,  0.18457031, -0.08398438,  0.18554688,  0.11669922,\n",
       "        0.02758789, -0.04760742,  0.17871094,  0.06542969, -0.03540039,\n",
       "        0.22949219,  0.02697754, -0.09765625,  0.26953125,  0.08349609,\n",
       "       -0.13085938, -0.10107422, -0.00738525,  0.07128906,  0.14941406,\n",
       "       -0.20605469,  0.18066406, -0.15820312,  0.05932617,  0.28710938,\n",
       "       -0.04663086,  0.15136719,  0.4921875 , -0.27539062,  0.05615234], dtype=float32)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmodel['cat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  5.12695312e-02,  -2.23388672e-02,  -1.72851562e-01,\n",
       "         1.61132812e-01,  -8.44726562e-02,   5.73730469e-02,\n",
       "         5.85937500e-02,  -8.25195312e-02,  -1.53808594e-02,\n",
       "        -6.34765625e-02,   1.79687500e-01,  -4.23828125e-01,\n",
       "        -2.25830078e-02,  -1.66015625e-01,  -2.51464844e-02,\n",
       "         1.07421875e-01,  -1.99218750e-01,   1.59179688e-01,\n",
       "        -1.87500000e-01,  -1.20117188e-01,   1.55273438e-01,\n",
       "        -9.91210938e-02,   1.42578125e-01,  -1.64062500e-01,\n",
       "        -8.93554688e-02,   2.00195312e-01,  -1.49414062e-01,\n",
       "         3.20312500e-01,   3.28125000e-01,   2.44140625e-02,\n",
       "        -9.71679688e-02,  -8.20312500e-02,  -3.63769531e-02,\n",
       "        -8.59375000e-02,  -9.86328125e-02,   7.78198242e-03,\n",
       "        -1.34277344e-02,   5.27343750e-02,   1.48437500e-01,\n",
       "         3.33984375e-01,   1.66015625e-02,  -2.12890625e-01,\n",
       "        -1.50756836e-02,   5.24902344e-02,  -1.07421875e-01,\n",
       "        -8.88671875e-02,   2.49023438e-01,  -7.03125000e-02,\n",
       "        -1.59912109e-02,   7.56835938e-02,  -7.03125000e-02,\n",
       "         1.19140625e-01,   2.29492188e-01,   1.41601562e-02,\n",
       "         1.15234375e-01,   7.50732422e-03,   2.75390625e-01,\n",
       "        -2.44140625e-01,   2.96875000e-01,   3.49121094e-02,\n",
       "         2.42187500e-01,   1.35742188e-01,   1.42578125e-01,\n",
       "         1.75781250e-02,   2.92968750e-02,  -1.21582031e-01,\n",
       "         2.28271484e-02,  -4.76074219e-02,  -1.55273438e-01,\n",
       "         3.14331055e-03,   3.45703125e-01,   1.22558594e-01,\n",
       "        -1.95312500e-01,   8.10546875e-02,  -6.83593750e-02,\n",
       "        -1.47094727e-02,   2.14843750e-01,  -1.21093750e-01,\n",
       "         1.57226562e-01,  -2.07031250e-01,   1.36718750e-01,\n",
       "        -1.29882812e-01,   5.29785156e-02,  -2.71484375e-01,\n",
       "        -2.98828125e-01,  -1.84570312e-01,  -2.29492188e-01,\n",
       "         1.19140625e-01,   1.53198242e-02,  -2.61718750e-01,\n",
       "        -1.23046875e-01,  -1.86767578e-02,  -6.49414062e-02,\n",
       "        -8.15429688e-02,   7.86132812e-02,  -3.53515625e-01,\n",
       "         5.24902344e-02,  -2.45361328e-02,  -5.43212891e-03,\n",
       "        -2.08984375e-01,  -2.10937500e-01,  -1.79687500e-01,\n",
       "         2.42187500e-01,   2.57812500e-01,   1.37695312e-01,\n",
       "        -2.10937500e-01,  -2.17285156e-02,  -1.38671875e-01,\n",
       "         1.84326172e-02,  -1.23901367e-02,  -1.59179688e-01,\n",
       "         1.61132812e-01,   2.08007812e-01,   1.03027344e-01,\n",
       "         9.81445312e-02,  -6.83593750e-02,  -8.72802734e-03,\n",
       "        -2.89062500e-01,  -2.14843750e-01,  -1.14257812e-01,\n",
       "        -2.21679688e-01,   4.12597656e-02,  -3.12500000e-01,\n",
       "        -5.59082031e-02,  -9.76562500e-02,   5.81054688e-02,\n",
       "        -4.05273438e-02,  -1.73828125e-01,   1.64062500e-01,\n",
       "        -2.53906250e-01,  -1.54296875e-01,  -2.31933594e-02,\n",
       "        -2.38281250e-01,   2.07519531e-02,  -2.73437500e-01,\n",
       "         3.90625000e-03,   1.13769531e-01,  -1.73828125e-01,\n",
       "         2.57812500e-01,   2.35351562e-01,   5.22460938e-02,\n",
       "         6.83593750e-02,  -1.75781250e-01,   1.60156250e-01,\n",
       "        -5.98907471e-04,   5.98144531e-02,  -2.11914062e-01,\n",
       "        -5.54199219e-02,  -7.51953125e-02,  -3.06640625e-01,\n",
       "         4.27734375e-01,   5.32226562e-02,  -2.08984375e-01,\n",
       "        -5.71289062e-02,  -2.09960938e-01,   3.29589844e-02,\n",
       "         1.05468750e-01,  -1.50390625e-01,  -9.37500000e-02,\n",
       "         1.16699219e-01,   6.44531250e-02,   2.80761719e-02,\n",
       "         2.41210938e-01,  -1.25976562e-01,  -1.00585938e-01,\n",
       "        -1.22680664e-02,  -3.26156616e-04,   1.58691406e-02,\n",
       "         1.27929688e-01,  -3.32031250e-02,   4.07714844e-02,\n",
       "        -1.31835938e-01,   9.81445312e-02,   1.74804688e-01,\n",
       "        -2.36328125e-01,   5.17578125e-02,   1.83593750e-01,\n",
       "         2.42919922e-02,  -4.31640625e-01,   2.46093750e-01,\n",
       "        -3.03955078e-02,  -2.47802734e-02,  -1.17187500e-01,\n",
       "         1.61132812e-01,  -5.71289062e-02,   1.16577148e-02,\n",
       "         2.81250000e-01,   4.27734375e-01,   4.56542969e-02,\n",
       "         1.01074219e-01,  -3.95507812e-02,   1.77001953e-02,\n",
       "        -8.98437500e-02,   1.35742188e-01,   2.08007812e-01,\n",
       "         1.88476562e-01,  -1.52343750e-01,  -2.37304688e-01,\n",
       "        -1.90429688e-01,   7.12890625e-02,  -2.46093750e-01,\n",
       "        -2.61718750e-01,  -2.34375000e-01,  -1.45507812e-01,\n",
       "        -1.17187500e-02,  -1.50390625e-01,  -1.13281250e-01,\n",
       "         1.82617188e-01,   2.63671875e-01,  -1.37695312e-01,\n",
       "        -4.58984375e-01,  -4.68750000e-02,  -1.26953125e-01,\n",
       "        -4.22363281e-02,  -1.66992188e-01,   1.26953125e-01,\n",
       "         2.59765625e-01,  -2.44140625e-01,  -2.19726562e-01,\n",
       "        -8.69140625e-02,   1.59179688e-01,  -3.78417969e-02,\n",
       "         8.97216797e-03,  -2.77343750e-01,  -1.04980469e-01,\n",
       "        -1.75781250e-01,   2.28515625e-01,  -2.70996094e-02,\n",
       "         2.85156250e-01,  -2.73437500e-01,   1.61132812e-02,\n",
       "         5.90820312e-02,  -2.39257812e-01,   1.77734375e-01,\n",
       "        -1.34765625e-01,   1.38671875e-01,   3.53515625e-01,\n",
       "         1.22070312e-01,   1.43554688e-01,   9.22851562e-02,\n",
       "         2.29492188e-01,  -3.00781250e-01,  -4.88281250e-02,\n",
       "        -1.79687500e-01,   2.96875000e-01,   1.75781250e-01,\n",
       "         4.80957031e-02,  -3.38745117e-03,   7.91015625e-02,\n",
       "        -2.38281250e-01,  -2.31445312e-01,   1.66015625e-01,\n",
       "        -2.13867188e-01,  -7.03125000e-02,  -7.56835938e-02,\n",
       "         1.96289062e-01,  -1.29882812e-01,  -1.05957031e-01,\n",
       "        -3.53515625e-01,  -1.16699219e-01,  -5.10253906e-02,\n",
       "         3.39355469e-02,  -1.43554688e-01,  -3.90625000e-03,\n",
       "         1.73828125e-01,  -9.96093750e-02,  -1.66015625e-01,\n",
       "        -8.54492188e-02,  -3.82812500e-01,   5.90820312e-02,\n",
       "        -6.22558594e-02,   8.83789062e-02,  -8.88671875e-02,\n",
       "         3.28125000e-01,   6.83593750e-02,  -1.91406250e-01,\n",
       "        -8.35418701e-04,   1.04003906e-01,   1.52343750e-01,\n",
       "        -1.53350830e-03,   4.16015625e-01,  -3.32031250e-02,\n",
       "         1.49414062e-01,   2.42187500e-01,  -1.76757812e-01,\n",
       "        -4.93164062e-02,  -1.24511719e-01,   1.25976562e-01,\n",
       "         1.74804688e-01,   2.81250000e-01,  -1.80664062e-01,\n",
       "         1.03027344e-01,  -2.75390625e-01,   2.61718750e-01,\n",
       "         2.46093750e-01,  -4.71191406e-02,   6.25000000e-02,\n",
       "         4.16015625e-01,  -3.55468750e-01,   2.22656250e-01], dtype=float32)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmodel['dog']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.19140625, -0.04296875,  0.27539062,  0.00488281, -0.3203125 ,\n",
       "        0.08203125,  0.05566406, -0.03613281, -0.31445312,  0.10693359,\n",
       "       -0.359375  ,  0.29882812,  0.02331543,  0.05517578, -0.140625  ,\n",
       "        0.1953125 , -0.23632812, -0.22167969, -0.06542969, -0.3359375 ,\n",
       "        0.25195312, -0.09326172,  0.54296875,  0.11328125, -0.28710938,\n",
       "       -0.12011719, -0.11181641,  0.20996094, -0.33203125,  0.30273438,\n",
       "       -0.3359375 , -0.12255859,  0.12890625, -0.28515625, -0.04223633,\n",
       "        0.25585938,  0.3203125 ,  0.07177734,  0.19042969, -0.01379395,\n",
       "        0.16992188, -0.22460938,  0.5078125 ,  0.08398438, -0.07519531,\n",
       "       -0.06396484,  0.05371094,  0.34570312,  0.46289062, -0.16699219,\n",
       "       -0.30664062,  0.15234375, -0.09765625, -0.26171875, -0.14160156,\n",
       "        0.2265625 ,  0.49609375, -0.10791016, -0.08447266,  0.234375  ,\n",
       "        0.04931641, -0.07128906,  0.05273438, -0.11914062,  0.09814453,\n",
       "        0.11181641, -0.13574219, -0.46875   ,  0.26171875,  0.12158203,\n",
       "        0.31445312,  0.05810547,  0.0703125 , -0.10107422, -0.27734375,\n",
       "       -0.16796875, -0.07128906, -0.08007812,  0.07226562, -0.1484375 ,\n",
       "        0.22949219,  0.03686523, -0.03857422,  0.00616455, -0.12255859,\n",
       "       -0.01940918, -0.0625    ,  0.26953125,  0.34179688, -0.00427246,\n",
       "       -0.49023438, -0.38867188, -0.24316406,  0.12304688,  0.07421875,\n",
       "       -0.25195312,  0.14941406, -0.265625  ,  0.30859375, -0.05834961,\n",
       "       -0.19726562, -0.14941406,  0.01031494, -0.07275391, -0.23632812,\n",
       "       -0.1484375 ,  0.3046875 , -0.10351562,  0.69140625, -0.29492188,\n",
       "       -0.25976562, -0.29296875,  0.31445312, -0.11083984, -0.5       ,\n",
       "       -0.04443359,  0.10058594,  0.04858398, -0.0625    ,  0.02001953,\n",
       "        0.08837891, -0.10058594, -0.00113678,  0.390625  ,  0.16894531,\n",
       "        0.01318359,  0.15625   ,  0.10595703,  0.29296875,  0.18457031,\n",
       "        0.13867188,  0.15820312, -0.38671875,  0.20214844,  0.17285156,\n",
       "        0.578125  , -0.05029297, -0.34179688,  0.46875   ,  0.34765625,\n",
       "        0.23535156, -0.20996094, -0.32226562, -0.26367188, -0.02160645,\n",
       "        0.50390625, -0.31054688, -0.2265625 ,  0.31640625, -0.14550781,\n",
       "        0.05639648, -0.18847656,  0.05126953,  0.73828125,  0.04174805,\n",
       "       -0.02392578, -0.375     , -0.18847656, -0.0625    ,  0.50390625,\n",
       "       -0.02844238, -0.05615234,  0.26367188, -0.0612793 ,  0.06030273,\n",
       "       -0.171875  , -0.00866699,  0.20019531, -0.03173828, -0.18164062,\n",
       "        0.40429688, -0.03710938, -0.03515625,  0.18261719, -0.47070312,\n",
       "       -0.07275391,  0.32617188, -0.17285156, -0.22265625,  0.27929688,\n",
       "       -0.57421875,  0.07275391,  0.20898438, -0.30273438,  0.19335938,\n",
       "        0.04907227, -0.15820312, -0.17675781,  0.18066406,  0.42773438,\n",
       "       -0.25390625,  0.29101562, -0.04760742, -0.28710938, -0.08837891,\n",
       "        0.28710938, -0.3828125 , -0.13574219,  0.05297852, -0.22265625,\n",
       "       -0.09179688, -0.06738281, -0.53125   ,  0.06933594,  0.02514648,\n",
       "        0.04443359, -0.18457031, -0.31054688,  0.02856445,  0.16992188,\n",
       "        0.01196289, -0.12109375,  0.00430298,  0.171875  ,  0.06640625,\n",
       "       -0.02954102, -0.39453125,  0.515625  ,  0.2109375 ,  0.03637695,\n",
       "       -0.390625  , -0.04980469, -0.13378906, -0.19140625, -0.34375   ,\n",
       "       -0.21289062,  0.375     ,  0.03039551,  0.1796875 , -0.37109375,\n",
       "        0.07763672, -0.07666016,  0.07910156, -0.15234375, -0.15429688,\n",
       "       -0.15039062,  0.12304688,  0.1796875 , -0.19335938,  0.125     ,\n",
       "        0.17285156,  0.03173828,  0.24121094, -0.34765625, -0.15136719,\n",
       "       -0.09619141,  0.09326172,  0.13964844, -0.20410156, -0.3671875 ,\n",
       "        0.02368164, -0.24804688,  0.17578125,  0.22460938,  0.18652344,\n",
       "       -0.44921875, -0.1640625 , -0.08837891,  0.1484375 , -0.0480957 ,\n",
       "        0.23144531, -0.06176758, -0.53125   , -0.23730469, -0.26953125,\n",
       "        0.02770996, -0.09667969, -0.05249023, -0.00543213, -0.05981445,\n",
       "        0.3203125 ,  0.34570312, -0.06591797, -0.05786133,  0.54296875,\n",
       "       -0.17675781, -0.05810547, -0.0378418 , -0.14550781, -0.265625  ,\n",
       "       -0.01153564,  0.3515625 ,  0.2421875 ,  0.24023438,  0.01025391,\n",
       "       -0.31445312, -0.20410156, -0.02172852, -0.21386719,  0.52734375,\n",
       "        0.12451172,  0.16113281, -0.46289062,  0.05126953, -0.02648926,\n",
       "       -0.546875  ,  0.30859375, -0.0534668 , -0.34765625,  0.3984375 ], dtype=float32)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmodel['spatula']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.76094570897822089"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmodel.similarity('cat', 'dog')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.12412612600429632"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmodel.similarity('cat', 'spatula')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models.doc2vec import TaggedDocument\n",
    "from gensim.models import Doc2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_words(sent):\n",
    "    sent = sent.lower()\n",
    "    sent = re.sub(r'<[^>]+>', ' ', sent) # strip html tags\n",
    "    sent = re.sub(r'(\\w)\\'(\\w)', '\\1\\2', sent) # remove apostrophes\n",
    "    sent = re.sub(r'\\W', ' ', sent) # remove punctuation\n",
    "    sent = re.sub(r'\\s+', ' ', sent) # remove repeated spaces\n",
    "    sent = sent.strip()\n",
    "    return sent.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# unsupervised training data\n",
    "import re\n",
    "import os\n",
    "unsup_sentences = []\n",
    "\n",
    "# source: http://ai.stanford.edu/~amaas/data/sentiment/, data from IMDB\n",
    "for dirname in [\"train/pos\", \"train/neg\", \"train/unsup\", \"test/pos\", \"test/neg\"]:\n",
    "    for fname in sorted(os.listdir(\"aclImdb/\" + dirname)):\n",
    "        if fname[-4:] == '.txt':\n",
    "            with open(\"aclImdb/\" + dirname + \"/\" + fname, encoding='UTF-8') as f:\n",
    "                sent = f.read()\n",
    "                words = extract_words(sent)\n",
    "                unsup_sentences.append(TaggedDocument(words, [dirname + \"/\" + fname]))\n",
    "\n",
    "# source: http://www.cs.cornell.edu/people/pabo/movie-review-data/\n",
    "for dirname in [\"review_polarity/txt_sentoken/pos\", \"review_polarity/txt_sentoken/neg\"]:\n",
    "    for fname in sorted(os.listdir(dirname)):\n",
    "        if fname[-4:] == '.txt':\n",
    "            with open(dirname + \"/\" + fname, encoding='UTF-8') as f:\n",
    "                for i, sent in enumerate(f):\n",
    "                    words = extract_words(sent)\n",
    "                    unsup_sentences.append(TaggedDocument(words, [\"%s/%s-%d\" % (dirname, fname, i)]))\n",
    "                \n",
    "# source: https://nlp.stanford.edu/sentiment/, data from Rotten Tomatoes\n",
    "with open(\"stanfordSentimentTreebank/original_rt_snippets.txt\", encoding='UTF-8') as f:\n",
    "    for i, line in enumerate(f):\n",
    "        words = extract_words(sent)\n",
    "        unsup_sentences.append(TaggedDocument(words, [\"rt-%d\" % i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "175325"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(unsup_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[TaggedDocument(words=['bromwell', 'high', 'is', 'a', 'cartoon', 'comedy', 'it', 'ran', 'at', 'the', 'same', 'time', 'as', 'some', 'other', 'programs', 'about', 'school', 'life', 'such', 'as', 'teachers', 'my', '35', 'years', 'in', 'the', 'teaching', 'profession', 'lead', 'me', 'to', 'believe', 'that', 'bromwell', 'hig', 'satire', 'is', 'much', 'closer', 'to', 'reality', 'than', 'is', 'teachers', 'the', 'scramble', 'to', 'survive', 'financially', 'the', 'insightful', 'students', 'who', 'can', 'see', 'right', 'through', 'their', 'pathetic', 'teachers', 'pomp', 'the', 'pettiness', 'of', 'the', 'whole', 'situation', 'all', 'remind', 'me', 'of', 'the', 'schools', 'i', 'knew', 'and', 'their', 'students', 'when', 'i', 'saw', 'the', 'episode', 'in', 'which', 'a', 'student', 'repeatedly', 'tried', 'to', 'burn', 'down', 'the', 'school', 'i', 'immediately', 'recalled', 'at', 'high', 'a', 'classic', 'line', 'inspector', 'here', 'to', 'sack', 'one', 'of', 'your', 'teachers', 'student', 'welcome', 'to', 'bromwell', 'high', 'i', 'expect', 'that', 'many', 'adults', 'of', 'my', 'age', 'think', 'that', 'bromwell', 'high', 'is', 'far', 'fetched', 'what', 'a', 'pity', 'that', 'it', 'is'], tags=['train/pos/0_9.txt']),\n",
       " TaggedDocument(words=['homelessness', 'or', 'houselessness', 'as', 'george', 'carlin', 'stated', 'has', 'been', 'an', 'issue', 'for', 'years', 'but', 'never', 'a', 'plan', 'to', 'help', 'those', 'on', 'the', 'street', 'that', 'were', 'once', 'considered', 'human', 'who', 'did', 'everything', 'from', 'going', 'to', 'school', 'work', 'or', 'vote', 'for', 'the', 'matter', 'most', 'people', 'think', 'of', 'the', 'homeless', 'as', 'just', 'a', 'lost', 'cause', 'while', 'worrying', 'about', 'things', 'such', 'as', 'racism', 'the', 'war', 'on', 'iraq', 'pressuring', 'kids', 'to', 'succeed', 'technology', 'the', 'elections', 'inflation', 'or', 'worrying', 'if', 'the', 'l', 'be', 'next', 'to', 'end', 'up', 'on', 'the', 'streets', 'but', 'what', 'if', 'you', 'were', 'given', 'a', 'bet', 'to', 'live', 'on', 'the', 'streets', 'for', 'a', 'month', 'without', 'the', 'luxuries', 'you', 'once', 'had', 'from', 'a', 'home', 'the', 'entertainment', 'sets', 'a', 'bathroom', 'pictures', 'on', 'the', 'wall', 'a', 'computer', 'and', 'everything', 'you', 'once', 'treasure', 'to', 'see', 'what', 'i', 'like', 'to', 'be', 'homeless', 'that', 'is', 'goddard', 'bol', 'lesson', 'mel', 'brooks', 'who', 'directs', 'who', 'stars', 'as', 'bolt', 'plays', 'a', 'rich', 'man', 'who', 'has', 'everything', 'in', 'the', 'world', 'until', 'deciding', 'to', 'make', 'a', 'bet', 'with', 'a', 'sissy', 'rival', 'jeffery', 'tambor', 'to', 'see', 'if', 'he', 'can', 'live', 'in', 'the', 'streets', 'for', 'thirty', 'days', 'without', 'the', 'luxuries', 'if', 'bolt', 'succeeds', 'he', 'can', 'do', 'what', 'he', 'wants', 'with', 'a', 'future', 'project', 'of', 'making', 'more', 'buildings', 'the', 'be', 'on', 'where', 'bolt', 'is', 'thrown', 'on', 'the', 'street', 'with', 'a', 'bracelet', 'on', 'his', 'leg', 'to', 'monitor', 'his', 'every', 'move', 'where', 'he', 'ca', 'step', 'off', 'the', 'sidewalk', 'h', 'given', 'the', 'nickname', 'pepto', 'by', 'a', 'vagrant', 'after', 'i', 'written', 'on', 'his', 'forehead', 'where', 'bolt', 'meets', 'other', 'characters', 'including', 'a', 'woman', 'by', 'the', 'name', 'of', 'molly', 'lesley', 'ann', 'warren', 'an', 'ex', 'dancer', 'who', 'got', 'divorce', 'before', 'losing', 'her', 'home', 'and', 'her', 'pals', 'sailor', 'howard', 'morris', 'and', 'fumes', 'teddy', 'wilson', 'who', 'are', 'already', 'used', 'to', 'the', 'streets', 'the', 'e', 'survivors', 'bolt', 'is', 'h', 'not', 'used', 'to', 'reaching', 'mutual', 'agreements', 'like', 'he', 'once', 'did', 'when', 'being', 'rich', 'where', 'i', 'fight', 'or', 'flight', 'kill', 'or', 'be', 'killed', 'while', 'the', 'love', 'connection', 'between', 'molly', 'and', 'bolt', 'was', 'necessary', 'to', 'plot', 'i', 'found', 'life', 'stinks', 'to', 'be', 'one', 'of', 'mel', 'brooks', 'observant', 'films', 'where', 'prior', 'to', 'being', 'a', 'comedy', 'it', 'shows', 'a', 'tender', 'side', 'compared', 'to', 'his', 'slapstick', 'work', 'such', 'as', 'blazing', 'saddles', 'young', 'frankenstein', 'or', 'spaceballs', 'for', 'the', 'matter', 'to', 'show', 'what', 'i', 'like', 'having', 'something', 'valuable', 'before', 'losing', 'it', 'the', 'next', 'day', 'or', 'on', 'the', 'other', 'hand', 'making', 'a', 'stupid', 'bet', 'like', 'all', 'rich', 'people', 'do', 'when', 'they', 'do', 'know', 'what', 'to', 'do', 'with', 'their', 'money', 'maybe', 'they', 'should', 'give', 'it', 'to', 'the', 'homeless', 'instead', 'of', 'using', 'it', 'like', 'monopoly', 'money', 'or', 'maybe', 'this', 'film', 'will', 'inspire', 'you', 'to', 'help', 'others'], tags=['train/pos/10000_8.txt']),\n",
       " TaggedDocument(words=['brilliant', 'over', 'acting', 'by', 'lesley', 'ann', 'warren', 'best', 'dramatic', 'hobo', 'lady', 'i', 'have', 'ever', 'seen', 'and', 'love', 'scenes', 'in', 'clothes', 'warehouse', 'are', 'second', 'to', 'none', 'the', 'corn', 'on', 'face', 'is', 'a', 'classic', 'as', 'good', 'as', 'anything', 'in', 'blazing', 'saddles', 'the', 'take', 'on', 'lawyers', 'is', 'also', 'superb', 'after', 'being', 'accused', 'of', 'being', 'a', 'turncoat', 'selling', 'out', 'his', 'boss', 'and', 'being', 'dishonest', 'the', 'lawyer', 'of', 'pepto', 'bolt', 'shrugs', 'indifferently', 'a', 'lawyer', 'he', 'says', 'three', 'funny', 'words', 'jeffrey', 'tambor', 'a', 'favorite', 'from', 'the', 'later', 'larry', 'sanders', 'show', 'is', 'fantastic', 'here', 'too', 'as', 'a', 'mad', 'millionaire', 'who', 'wants', 'to', 'crush', 'the', 'ghetto', 'his', 'character', 'is', 'more', 'malevolent', 'than', 'usual', 'the', 'hospital', 'scene', 'and', 'the', 'scene', 'where', 'the', 'homeless', 'invade', 'a', 'demolition', 'site', 'are', 'all', 'time', 'classics', 'look', 'for', 'the', 'legs', 'scene', 'and', 'the', 'two', 'big', 'diggers', 'fighting', 'one', 'bleeds', 'this', 'movie', 'gets', 'better', 'each', 'time', 'i', 'see', 'it', 'which', 'is', 'quite', 'often'], tags=['train/pos/10001_10.txt']),\n",
       " TaggedDocument(words=['this', 'is', 'easily', 'the', 'most', 'underrated', 'film', 'inn', 'the', 'brooks', 'cannon', 'sure', 'its', 'flawed', 'it', 'does', 'not', 'give', 'a', 'realistic', 'view', 'of', 'homelessness', 'unlike', 'say', 'how', 'citizen', 'kane', 'gave', 'a', 'realistic', 'view', 'of', 'lounge', 'singers', 'or', 'titanic', 'gave', 'a', 'realistic', 'view', 'of', 'italians', 'you', 'idiots', 'many', 'of', 'the', 'jokes', 'fall', 'flat', 'but', 'still', 'this', 'film', 'is', 'very', 'lovable', 'in', 'a', 'way', 'many', 'comedies', 'are', 'not', 'and', 'to', 'pull', 'that', 'off', 'in', 'a', 'story', 'about', 'some', 'of', 'the', 'most', 'traditionally', 'reviled', 'members', 'of', 'society', 'is', 'truly', 'impressive', 'its', 'not', 'the', 'fisher', 'king', 'but', 'its', 'not', 'crap', 'either', 'my', 'only', 'complaint', 'is', 'that', 'brooks', 'should', 'have', 'cast', 'someone', 'else', 'in', 'the', 'lead', 'i', 'love', 'mel', 'as', 'a', 'director', 'and', 'writer', 'not', 'so', 'much', 'as', 'a', 'lead'], tags=['train/pos/10002_7.txt']),\n",
       " TaggedDocument(words=['this', 'is', 'not', 'the', 'typical', 'mel', 'brooks', 'film', 'it', 'was', 'much', 'less', 'slapstick', 'than', 'most', 'of', 'his', 'movies', 'and', 'actually', 'had', 'a', 'plot', 'that', 'was', 'followable', 'leslie', 'ann', 'warren', 'made', 'the', 'movie', 'she', 'is', 'such', 'a', 'fantastic', 'under', 'rated', 'actress', 'there', 'were', 'some', 'moments', 'that', 'could', 'have', 'been', 'fleshed', 'out', 'a', 'bit', 'more', 'and', 'some', 'scenes', 'that', 'could', 'probably', 'have', 'been', 'cut', 'to', 'make', 'the', 'room', 'to', 'do', 'so', 'but', 'all', 'in', 'all', 'this', 'is', 'worth', 'the', 'price', 'to', 'rent', 'and', 'see', 'it', 'the', 'acting', 'was', 'good', 'overall', 'brooks', 'himself', 'did', 'a', 'good', 'job', 'without', 'his', 'characteristic', 'speaking', 'to', 'directly', 'to', 'the', 'audience', 'again', 'warren', 'was', 'the', 'best', 'actor', 'in', 'the', 'movie', 'but', 'fume', 'and', 'sailor', 'both', 'played', 'their', 'parts', 'well'], tags=['train/pos/10003_8.txt']),\n",
       " TaggedDocument(words=['this', 'is', 'the', 'comedic', 'robin', 'williams', 'nor', 'is', 'it', 'the', 'quirky', 'insane', 'robin', 'williams', 'of', 'recent', 'thriller', 'fame', 'this', 'is', 'a', 'hybrid', 'of', 'the', 'classic', 'drama', 'without', 'over', 'dramatization', 'mixed', 'with', 'robi', 'new', 'love', 'of', 'the', 'thriller', 'but', 'this', 'is', 'a', 'thriller', 'per', 'se', 'this', 'is', 'more', 'a', 'mystery', 'suspense', 'vehicle', 'through', 'which', 'williams', 'attempts', 'to', 'locate', 'a', 'sick', 'boy', 'and', 'his', 'keeper', 'also', 'starring', 'sandra', 'oh', 'and', 'rory', 'culkin', 'this', 'suspense', 'drama', 'plays', 'pretty', 'much', 'like', 'a', 'news', 'report', 'until', 'willia', 'character', 'gets', 'close', 'to', 'achieving', 'his', 'goal', 'i', 'must', 'say', 'that', 'i', 'was', 'highly', 'entertained', 'though', 'this', 'movie', 'fails', 'to', 'teach', 'guide', 'inspect', 'or', 'amuse', 'it', 'felt', 'more', 'like', 'i', 'was', 'watching', 'a', 'guy', 'williams', 'as', 'he', 'was', 'actually', 'performing', 'the', 'actions', 'from', 'a', 'third', 'person', 'perspective', 'in', 'other', 'words', 'it', 'felt', 'real', 'and', 'i', 'was', 'able', 'to', 'subscribe', 'to', 'the', 'premise', 'of', 'the', 'story', 'all', 'in', 'all', 'i', 'worth', 'a', 'watch', 'though', 'i', 'definitely', 'not', 'friday', 'saturday', 'night', 'fare', 'it', 'rates', 'a', '7', '7', '10', 'from', 'the', 'fiend'], tags=['train/pos/10004_8.txt']),\n",
       " TaggedDocument(words=['yes', 'its', 'an', 'art', 'to', 'successfully', 'make', 'a', 'slow', 'paced', 'thriller', 'the', 'story', 'unfolds', 'in', 'nice', 'volumes', 'while', 'you', 'do', 'even', 'notice', 'it', 'happening', 'fine', 'performance', 'by', 'robin', 'williams', 'the', 'sexuality', 'angles', 'in', 'the', 'film', 'can', 'seem', 'unnecessary', 'and', 'can', 'probably', 'affect', 'how', 'much', 'you', 'enjoy', 'the', 'film', 'however', 'the', 'core', 'plot', 'is', 'very', 'engaging', 'the', 'movie', 'does', 'rush', 'onto', 'you', 'and', 'still', 'grips', 'you', 'enough', 'to', 'keep', 'you', 'wondering', 'the', 'direction', 'is', 'good', 'use', 'of', 'lights', 'to', 'achieve', 'desired', 'affects', 'of', 'suspense', 'and', 'unexpectedness', 'is', 'good', 'very', 'nice', '1', 'time', 'watch', 'if', 'you', 'are', 'looking', 'to', 'lay', 'back', 'and', 'hear', 'a', 'thrilling', 'short', 'story'], tags=['train/pos/10005_7.txt']),\n",
       " TaggedDocument(words=['in', 'this', 'critically', 'acclaimed', 'psychological', 'thriller', 'based', 'on', 'true', 'events', 'gabriel', 'robin', 'williams', 'a', 'celebrated', 'writer', 'and', 'late', 'night', 'talk', 'show', 'host', 'becomes', 'captivated', 'by', 'the', 'harrowing', 'story', 'of', 'a', 'young', 'listener', 'and', 'his', 'adoptive', 'mother', 'toni', 'collette', 'when', 'troubling', 'questions', 'arise', 'about', 'this', 'bo', 'story', 'however', 'gabriel', 'finds', 'himself', 'drawn', 'into', 'a', 'widening', 'mystery', 'that', 'hides', 'a', 'deadly', 'secret', 'according', 'to', 'fil', 'official', 'synopsis', 'you', 'really', 'should', 'stop', 'reading', 'these', 'comments', 'and', 'watch', 'the', 'film', 'now', 'the', 'how', 'did', 'he', 'lose', 'his', 'leg', 'ending', 'with', 'ms', 'collette', 'planning', 'her', 'new', 'life', 'should', 'be', 'chopped', 'off', 'and', 'sent', 'to', 'deleted', 'scenes', 'land', 'i', 'overkill', 'the', 'true', 'nature', 'of', 'her', 'physical', 'and', 'mental', 'ailments', 'should', 'be', 'obvious', 'by', 'the', 'time', 'mr', 'williams', 'returns', 'to', 'new', 'york', 'possibly', 'her', 'blindness', 'could', 'be', 'in', 'question', 'but', 'a', 'revelation', 'could', 'have', 'be', 'made', 'certain', 'in', 'either', 'the', 'highway', 'or', 'video', 'tape', 'scenes', 'the', 'film', 'would', 'benefit', 'from', 'a', 're', 'editing', 'how', 'about', 'a', 'directo', 'cut', 'williams', 'and', 'bobby', 'cannavale', 'as', 'jess', 'do', 'seem', 'initially', 'believable', 'as', 'a', 'couple', 'a', 'scene', 'or', 'two', 'establishing', 'their', 'relationship', 'might', 'have', 'helped', 'set', 'the', 'stage', 'otherwise', 'the', 'cast', 'is', 'exemplary', 'williams', 'offers', 'an', 'exceptionally', 'strong', 'characterization', 'and', 'not', 'a', 'gay', 'impersonation', 'sandra', 'oh', 'as', 'anna', 'joe', 'morton', 'as', 'ashe', 'and', 'rory', 'culkin', 'pete', 'logand', 'are', 'all', 'perfect', 'best', 'of', 'all', 'collett', 'donna', 'belongs', 'in', 'the', 'creepy', 'hall', 'of', 'fame', 'ms', 'oh', 'is', 'correct', 'in', 'saying', 'collette', 'might', 'be', 'you', 'know', 'like', 'that', 'guy', 'from', 'psycho', 'there', 'have', 'been', 'several', 'years', 'when', 'organizations', 'giving', 'acting', 'awards', 'seemed', 'to', 'reach', 'for', 'women', 'due', 'to', 'a', 'slighter', 'dispersion', 'of', 'roles', 'certainly', 'they', 'could', 'have', 'noticed', 'collette', 'with', 'some', 'award', 'consideration', 'she', 'is', 'that', 'good', 'and', 'director', 'patrick', 'stettner', 'definitely', 'evokes', 'hitchcock', 'he', 'even', 'makes', 'getting', 'a', 'sandwich', 'from', 'a', 'vending', 'machine', 'suspenseful', 'finally', 'writers', 'stettner', 'armistead', 'maupin', 'and', 'terry', 'anderson', 'deserve', 'gratitude', 'from', 'flight', 'attendants', 'everywhere', 'the', 'night', 'listener', '1', '21', '06', 'patrick', 'stettner', 'robin', 'williams', 'toni', 'collette', 'sandra', 'oh', 'rory', 'culkin'], tags=['train/pos/10006_7.txt']),\n",
       " TaggedDocument(words=['the', 'night', 'listener', '2006', '1', '2', 'robin', 'williams', 'toni', 'collette', 'bobby', 'cannavale', 'rory', 'culkin', 'joe', 'morton', 'sandra', 'oh', 'john', 'cullum', 'lisa', 'emery', 'becky', 'ann', 'baker', 'dir', 'patrick', 'stettner', 'hitchcockian', 'suspenser', 'gives', 'williams', 'a', 'stand', 'out', 'low', 'key', 'performance', 'what', 'is', 'it', 'about', 'celebrities', 'and', 'fans', 'what', 'is', 'the', 'near', 'paranoia', 'one', 'associates', 'with', 'the', 'other', 'and', 'why', 'is', 'it', 'almost', 'the', 'norm', 'in', 'the', 'latest', 'derange', 'fan', 'scenario', 'based', 'on', 'true', 'events', 'no', 'less', 'williams', 'stars', 'as', 'a', 'talk', 'radio', 'personality', 'named', 'gabriel', 'no', 'one', 'who', 'reads', 'stories', 'h', 'penned', 'over', 'the', 'airwaves', 'and', 'has', 'accumulated', 'an', 'interesting', 'fan', 'in', 'the', 'form', 'of', 'a', 'young', 'boy', 'named', 'pete', 'logand', 'culkin', 'who', 'has', 'submitted', 'a', 'manuscript', 'about', 'the', 'travails', 'of', 'his', 'troubled', 'youth', 'to', 'no', 'on', 'editor', 'ashe', 'morton', 'who', 'gives', 'it', 'to', 'no', 'one', 'to', 'read', 'for', 'himself', 'no', 'one', 'is', 'naturally', 'disturbed', 'but', 'ultimately', 'intrigued', 'about', 'the', 'nightmarish', 'existence', 'of', 'pete', 'being', 'abducted', 'and', 'sexually', 'abused', 'for', 'years', 'until', 'he', 'was', 'finally', 'rescued', 'by', 'a', 'nurse', 'named', 'donna', 'collette', 'giving', 'an', 'excellent', 'performance', 'who', 'has', 'adopted', 'the', 'boy', 'but', 'her', 'correspondence', 'with', 'no', 'one', 'reveals', 'that', 'pete', 'is', 'dying', 'from', 'aids', 'naturally', 'no', 'one', 'wants', 'to', 'meet', 'the', 'fans', 'but', 'is', 'suddenly', 'in', 'doubt', 'to', 'their', 'possibly', 'devious', 'ulterior', 'motives', 'when', 'the', 'seed', 'is', 'planted', 'by', 'his', 'estranged', 'lover', 'jess', 'cannavale', 'whose', 'sudden', 'departure', 'from', 'their', 'new', 'york', 'city', 'apartment', 'has', 'no', 'one', 'in', 'an', 'emotional', 'tailspin', 'that', 'has', 'only', 'now', 'grown', 'into', 'a', 'tempest', 'in', 'a', 'teacup', 'when', 'he', 'decides', 'to', 'do', 'some', 'investigating', 'into', 'donna', 'and', 'pet', 'backgrounds', 'discovering', 'some', 'truths', 'that', 'he', 'did', 'anticipate', 'written', 'by', 'armistead', 'maupin', 'who', 'co', 'wrote', 'the', 'screenplay', 'with', 'his', 'former', 'lover', 'terry', 'anderson', 'and', 'the', 'fil', 'novice', 'director', 'stettner', 'and', 'based', 'on', 'a', 'true', 'story', 'about', 'a', 'fa', 'hoax', 'found', 'out', 'has', 'some', 'hitchcockian', 'moments', 'that', 'run', 'on', 'full', 'tilt', 'like', 'any', 'good', 'old', 'fashioned', 'pot', 'boiler', 'does', 'it', 'helps', 'that', 'williams', 'gives', 'a', 'stand', 'out', 'low', 'key', 'performance', 'as', 'the', 'conflicted', 'good', 'hearted', 'personality', 'who', 'genuinely', 'wants', 'to', 'believe', 'that', 'his', 'number', 'one', 'fan', 'is', 'in', 'fact', 'real', 'and', 'does', 'love', 'him', 'the', 'one', 'thing', 'that', 'has', 'escaped', 'his', 'own', 'reality', 'and', 'has', 'some', 'unsettling', 'dreadful', 'moments', 'with', 'the', 'creepy', 'collette', 'whose', 'one', 'physical', 'trait', 'i', 'will', 'leave', 'unmentioned', 'but', 'underlines', 'the', 'desperation', 'of', 'her', 'character', 'that', 'can', 'rattle', 'you', 'to', 'the', 'core', 'however', 'the', 'film', 'runs', 'out', 'of', 'gas', 'and', 'eventually', 'becomes', 'a', 'bit', 'repetitive', 'and', 'predictable', 'despite', 'a', 'finely', 'directed', 'piece', 'of', 'hoodwink', 'and', 'mystery', 'by', 'stettner', 'it', 'pays', 'to', 'listen', 'to', 'your', 'own', 'inner', 'voice', 'be', 'careful', 'of', 'what', 'you', 'hope', 'for'], tags=['train/pos/10007_7.txt']),\n",
       " TaggedDocument(words=['you', 'know', 'robin', 'williams', 'god', 'bless', 'him', 'is', 'constantly', 'shooting', 'himself', 'in', 'the', 'foot', 'lately', 'with', 'all', 'these', 'dumb', 'comedies', 'he', 'has', 'done', 'this', 'decade', 'with', 'perhaps', 'the', 'exception', 'of', 'death', 'to', 'smoochy', 'which', 'bombed', 'when', 'it', 'came', 'out', 'but', 'is', 'now', 'a', 'cult', 'classic', 'the', 'dramas', 'he', 'has', 'made', 'lately', 'have', 'been', 'fantastic', 'especially', 'insomnia', 'and', 'one', 'hour', 'photo', 'the', 'night', 'listener', 'despite', 'mediocre', 'reviews', 'and', 'a', 'quick', 'dvd', 'release', 'is', 'among', 'his', 'best', 'work', 'period', 'this', 'is', 'a', 'very', 'chilling', 'story', 'even', 'though', 'it', 'does', 'include', 'a', 'serial', 'killer', 'or', 'anyone', 'that', 'physically', 'dangerous', 'for', 'that', 'matter', 'the', 'concept', 'of', 'the', 'film', 'is', 'based', 'on', 'an', 'actual', 'case', 'of', 'fraud', 'that', 'still', 'has', 'yet', 'to', 'be', 'officially', 'confirmed', 'in', 'high', 'school', 'i', 'read', 'an', 'autobiography', 'by', 'a', 'child', 'named', 'anthony', 'godby', 'johnson', 'who', 'suffered', 'horrific', 'abuse', 'and', 'eventually', 'contracted', 'aids', 'as', 'a', 'result', 'i', 'was', 'moved', 'by', 'the', 'story', 'until', 'i', 'read', 'reports', 'online', 'that', 'johnson', 'may', 'not', 'actually', 'exist', 'when', 'i', 'saw', 'this', 'movie', 'the', 'confused', 'feelings', 'that', 'robin', 'williams', 'so', 'brilliantly', 'portrayed', 'resurfaced', 'in', 'my', 'mind', 'toni', 'collette', 'probably', 'gives', 'her', 'best', 'dramatic', 'performance', 'too', 'as', 'the', 'ultimately', 'sociopathic', 'caretaker', 'her', 'role', 'was', 'a', 'far', 'cry', 'from', 'those', 'she', 'had', 'in', 'movies', 'like', 'little', 'miss', 'sunshine', 'there', 'were', 'even', 'times', 'she', 'looked', 'into', 'the', 'camera', 'where', 'i', 'thought', 'she', 'was', 'staring', 'right', 'at', 'me', 'it', 'takes', 'a', 'good', 'actress', 'to', 'play', 'that', 'sort', 'of', 'role', 'and', 'i', 'this', 'understated', 'yet', 'well', 'reviewed', 'role', 'that', 'makes', 'toni', 'collette', 'probably', 'one', 'of', 'the', 'best', 'actresses', 'of', 'this', 'generation', 'not', 'to', 'have', 'even', 'been', 'nominated', 'for', 'an', 'academy', 'award', 'as', 'of', '2008', 'i', 'incredible', 'that', 'there', 'is', 'at', 'least', 'one', 'woman', 'in', 'this', 'world', 'who', 'is', 'like', 'this', 'and', 'i', 'scary', 'too', 'this', 'is', 'a', 'good', 'dark', 'film', 'that', 'i', 'highly', 'recommend', 'be', 'prepared', 'to', 'be', 'unsettled', 'though', 'because', 'this', 'movie', 'leaves', 'you', 'with', 'a', 'strange', 'feeling', 'at', 'the', 'end'], tags=['train/pos/10008_7.txt'])]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unsup_sentences[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "class PermuteSentences(object):\n",
    "    def __init__(self, sents):\n",
    "        self.sents = sents\n",
    "        \n",
    "    def __iter__(self):\n",
    "        shuffled = list(self.sents)\n",
    "        random.shuffle(shuffled)\n",
    "        for sent in shuffled:\n",
    "            yield sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-12 16:09:07,700 : WARNING : consider setting layer size to a multiple of 4 for greater performance\n",
      "2017-11-12 16:09:07,703 : INFO : collecting all words and their counts\n",
      "2017-11-12 16:09:08,097 : INFO : PROGRESS: at example #0, processed 0 words (0/s), 0 word types, 0 tags\n",
      "2017-11-12 16:09:08,837 : INFO : PROGRESS: at example #10000, processed 1389399 words (1883036/s), 44788 word types, 10000 tags\n",
      "2017-11-12 16:09:09,546 : INFO : PROGRESS: at example #20000, processed 2818609 words (2024079/s), 61270 word types, 20000 tags\n",
      "2017-11-12 16:09:10,234 : INFO : PROGRESS: at example #30000, processed 4225040 words (2051374/s), 72774 word types, 30000 tags\n",
      "2017-11-12 16:09:10,921 : INFO : PROGRESS: at example #40000, processed 5632885 words (2057947/s), 82225 word types, 40000 tags\n",
      "2017-11-12 16:09:11,602 : INFO : PROGRESS: at example #50000, processed 7040997 words (2074544/s), 90454 word types, 50000 tags\n",
      "2017-11-12 16:09:12,284 : INFO : PROGRESS: at example #60000, processed 8474509 words (2106160/s), 97853 word types, 60000 tags\n",
      "2017-11-12 16:09:12,983 : INFO : PROGRESS: at example #70000, processed 9855488 words (1983678/s), 104567 word types, 70000 tags\n",
      "2017-11-12 16:09:13,669 : INFO : PROGRESS: at example #80000, processed 11260351 words (2053644/s), 110705 word types, 80000 tags\n",
      "2017-11-12 16:09:14,369 : INFO : PROGRESS: at example #90000, processed 12697330 words (2059473/s), 116530 word types, 90000 tags\n",
      "2017-11-12 16:09:15,047 : INFO : PROGRESS: at example #100000, processed 14083612 words (2051503/s), 121947 word types, 100000 tags\n",
      "2017-11-12 16:09:15,744 : INFO : PROGRESS: at example #110000, processed 15473580 words (2001396/s), 127068 word types, 110000 tags\n",
      "2017-11-12 16:09:16,454 : INFO : PROGRESS: at example #120000, processed 16904319 words (2022334/s), 131927 word types, 120000 tags\n",
      "2017-11-12 16:09:17,162 : INFO : PROGRESS: at example #130000, processed 18314137 words (1997714/s), 136539 word types, 130000 tags\n",
      "2017-11-12 16:09:17,829 : INFO : PROGRESS: at example #140000, processed 19724275 words (2122862/s), 140919 word types, 140000 tags\n",
      "2017-11-12 16:09:18,549 : INFO : PROGRESS: at example #150000, processed 21141383 words (1975527/s), 145229 word types, 150000 tags\n",
      "2017-11-12 16:09:19,233 : INFO : PROGRESS: at example #160000, processed 22523806 words (2028082/s), 149161 word types, 160000 tags\n",
      "2017-11-12 16:09:19,933 : INFO : PROGRESS: at example #170000, processed 23959319 words (2055830/s), 153261 word types, 170000 tags\n",
      "2017-11-12 16:09:20,298 : INFO : collected 155374 word types and 175325 unique tags from a corpus of 175325 examples and 24693510 words\n",
      "2017-11-12 16:09:20,301 : INFO : Loading a fresh vocabulary\n",
      "2017-11-12 16:09:20,743 : INFO : min_count=5 retains 57730 unique words (37% of original 155374, drops 97644)\n",
      "2017-11-12 16:09:20,745 : INFO : min_count=5 leaves 24537325 word corpus (99% of original 24693510, drops 156185)\n",
      "2017-11-12 16:09:21,225 : INFO : deleting the raw counts dictionary of 155374 items\n",
      "2017-11-12 16:09:21,240 : INFO : sample=0.001 downsamples 47 most-common words\n",
      "2017-11-12 16:09:21,242 : INFO : downsampling leaves estimated 18578883 word corpus (75.7% of prior 24537325)\n",
      "2017-11-12 16:09:21,243 : INFO : estimated required memory for 57730 words and 50 dimensions: 145179000 bytes\n",
      "2017-11-12 16:09:21,393 : INFO : constructing a huffman tree from 57730 words\n",
      "2017-11-12 16:09:25,484 : INFO : built huffman tree with maximum node depth 22\n",
      "2017-11-12 16:09:25,806 : INFO : resetting layer weights\n",
      "2017-11-12 16:09:32,109 : INFO : training model with 3 workers on 57730 vocabulary and 50 features, using sg=1 hs=1 sample=0.001 negative=5 window=5\n",
      "2017-11-12 16:09:33,149 : INFO : PROGRESS: at 0.18% examples, 167132 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:34,178 : INFO : PROGRESS: at 0.53% examples, 236501 words/s, in_qsize 4, out_qsize 1\n",
      "2017-11-12 16:09:35,205 : INFO : PROGRESS: at 0.89% examples, 262097 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:36,227 : INFO : PROGRESS: at 1.23% examples, 274908 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:09:37,232 : INFO : PROGRESS: at 1.56% examples, 283730 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:38,272 : INFO : PROGRESS: at 1.89% examples, 287822 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:39,278 : INFO : PROGRESS: at 2.23% examples, 291300 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:40,291 : INFO : PROGRESS: at 2.55% examples, 292786 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:41,294 : INFO : PROGRESS: at 2.89% examples, 294756 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:42,295 : INFO : PROGRESS: at 3.22% examples, 296648 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:09:43,314 : INFO : PROGRESS: at 3.54% examples, 296973 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:44,369 : INFO : PROGRESS: at 3.88% examples, 297505 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:09:45,370 : INFO : PROGRESS: at 4.23% examples, 298742 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:46,392 : INFO : PROGRESS: at 4.57% examples, 299829 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:47,445 : INFO : PROGRESS: at 4.91% examples, 300596 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:48,448 : INFO : PROGRESS: at 5.25% examples, 301322 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:49,496 : INFO : PROGRESS: at 5.61% examples, 302084 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:50,525 : INFO : PROGRESS: at 5.94% examples, 302715 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:51,542 : INFO : PROGRESS: at 6.29% examples, 302993 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:52,542 : INFO : PROGRESS: at 6.63% examples, 303586 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:53,591 : INFO : PROGRESS: at 6.96% examples, 303643 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:09:54,629 : INFO : PROGRESS: at 7.31% examples, 304250 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:55,639 : INFO : PROGRESS: at 7.66% examples, 304225 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:09:56,658 : INFO : PROGRESS: at 7.99% examples, 304659 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:57,670 : INFO : PROGRESS: at 8.32% examples, 305150 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:09:58,701 : INFO : PROGRESS: at 8.65% examples, 305096 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:09:59,727 : INFO : PROGRESS: at 9.00% examples, 305426 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:00,732 : INFO : PROGRESS: at 9.33% examples, 305374 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:01,768 : INFO : PROGRESS: at 9.67% examples, 305295 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:02,768 : INFO : PROGRESS: at 10.02% examples, 305986 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:03,789 : INFO : PROGRESS: at 10.36% examples, 306273 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:04,794 : INFO : PROGRESS: at 10.70% examples, 306867 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:05,798 : INFO : PROGRESS: at 11.04% examples, 307044 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:06,811 : INFO : PROGRESS: at 11.38% examples, 307299 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:07,815 : INFO : PROGRESS: at 11.72% examples, 307602 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:08,831 : INFO : PROGRESS: at 12.06% examples, 307663 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:09,875 : INFO : PROGRESS: at 12.41% examples, 307657 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:10,877 : INFO : PROGRESS: at 12.76% examples, 308156 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:11,893 : INFO : PROGRESS: at 13.09% examples, 308183 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:12,934 : INFO : PROGRESS: at 13.43% examples, 308205 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:13,947 : INFO : PROGRESS: at 13.77% examples, 308424 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:14,970 : INFO : PROGRESS: at 14.10% examples, 308571 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:15,983 : INFO : PROGRESS: at 14.45% examples, 308771 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:16,989 : INFO : PROGRESS: at 14.79% examples, 309014 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:18,014 : INFO : PROGRESS: at 15.12% examples, 308967 words/s, in_qsize 5, out_qsize 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-12 16:10:19,029 : INFO : PROGRESS: at 15.44% examples, 308818 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:20,079 : INFO : PROGRESS: at 15.79% examples, 308916 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:21,116 : INFO : PROGRESS: at 16.13% examples, 309076 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:22,128 : INFO : PROGRESS: at 16.46% examples, 309106 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:23,180 : INFO : PROGRESS: at 16.81% examples, 309171 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:24,186 : INFO : PROGRESS: at 17.14% examples, 309106 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:25,200 : INFO : PROGRESS: at 17.48% examples, 309241 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:26,214 : INFO : PROGRESS: at 17.82% examples, 309239 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:27,226 : INFO : PROGRESS: at 18.15% examples, 309143 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:28,253 : INFO : PROGRESS: at 18.50% examples, 309226 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:29,288 : INFO : PROGRESS: at 18.84% examples, 309231 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:30,291 : INFO : PROGRESS: at 19.18% examples, 309160 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:31,300 : INFO : PROGRESS: at 19.52% examples, 309413 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:32,324 : INFO : PROGRESS: at 19.86% examples, 309366 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:33,354 : INFO : PROGRESS: at 20.02% examples, 306594 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:34,358 : INFO : PROGRESS: at 20.35% examples, 306591 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:35,375 : INFO : PROGRESS: at 20.69% examples, 306612 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:36,386 : INFO : PROGRESS: at 21.04% examples, 306555 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:37,419 : INFO : PROGRESS: at 21.38% examples, 306502 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:38,439 : INFO : PROGRESS: at 21.73% examples, 306631 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:39,464 : INFO : PROGRESS: at 22.05% examples, 306507 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:40,464 : INFO : PROGRESS: at 22.37% examples, 306608 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:41,484 : INFO : PROGRESS: at 22.71% examples, 306718 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:42,493 : INFO : PROGRESS: at 23.05% examples, 306778 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:43,517 : INFO : PROGRESS: at 23.38% examples, 306789 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:44,543 : INFO : PROGRESS: at 23.72% examples, 306758 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:45,547 : INFO : PROGRESS: at 24.06% examples, 306928 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:46,588 : INFO : PROGRESS: at 24.41% examples, 306864 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:47,627 : INFO : PROGRESS: at 24.76% examples, 306996 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:48,628 : INFO : PROGRESS: at 25.09% examples, 306989 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:49,633 : INFO : PROGRESS: at 25.42% examples, 307034 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:50,638 : INFO : PROGRESS: at 25.75% examples, 307090 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:51,664 : INFO : PROGRESS: at 26.10% examples, 307163 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:52,691 : INFO : PROGRESS: at 26.43% examples, 307139 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:53,722 : INFO : PROGRESS: at 26.76% examples, 307157 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:54,763 : INFO : PROGRESS: at 27.10% examples, 307172 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:55,783 : INFO : PROGRESS: at 27.45% examples, 307266 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:56,803 : INFO : PROGRESS: at 27.80% examples, 307362 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:10:57,820 : INFO : PROGRESS: at 28.14% examples, 307307 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:58,832 : INFO : PROGRESS: at 28.48% examples, 307422 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:10:59,841 : INFO : PROGRESS: at 28.80% examples, 307471 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:00,844 : INFO : PROGRESS: at 29.12% examples, 307445 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:01,856 : INFO : PROGRESS: at 29.46% examples, 307471 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:02,868 : INFO : PROGRESS: at 29.80% examples, 307582 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:03,871 : INFO : PROGRESS: at 30.13% examples, 307564 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:04,884 : INFO : PROGRESS: at 30.44% examples, 307504 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:05,884 : INFO : PROGRESS: at 30.77% examples, 307496 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:06,889 : INFO : PROGRESS: at 31.10% examples, 307561 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:07,894 : INFO : PROGRESS: at 31.44% examples, 307525 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:08,896 : INFO : PROGRESS: at 31.76% examples, 307517 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:09,906 : INFO : PROGRESS: at 32.08% examples, 307467 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:10,913 : INFO : PROGRESS: at 32.41% examples, 307504 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:11,931 : INFO : PROGRESS: at 32.75% examples, 307658 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:12,962 : INFO : PROGRESS: at 33.12% examples, 307995 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:13,977 : INFO : PROGRESS: at 33.49% examples, 308453 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:14,985 : INFO : PROGRESS: at 33.85% examples, 308706 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:15,999 : INFO : PROGRESS: at 34.23% examples, 309076 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:17,000 : INFO : PROGRESS: at 34.60% examples, 309467 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:18,019 : INFO : PROGRESS: at 34.97% examples, 309746 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:19,059 : INFO : PROGRESS: at 35.34% examples, 310077 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:20,094 : INFO : PROGRESS: at 35.72% examples, 310416 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:21,115 : INFO : PROGRESS: at 36.10% examples, 310731 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:22,144 : INFO : PROGRESS: at 36.48% examples, 311082 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:23,148 : INFO : PROGRESS: at 36.87% examples, 311431 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:24,157 : INFO : PROGRESS: at 37.24% examples, 311706 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:25,161 : INFO : PROGRESS: at 37.61% examples, 312043 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:26,179 : INFO : PROGRESS: at 37.99% examples, 312386 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:27,209 : INFO : PROGRESS: at 38.36% examples, 312642 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:28,236 : INFO : PROGRESS: at 38.74% examples, 312905 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:29,255 : INFO : PROGRESS: at 39.12% examples, 313235 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:30,274 : INFO : PROGRESS: at 39.49% examples, 313499 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:31,291 : INFO : PROGRESS: at 39.86% examples, 313762 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:32,305 : INFO : PROGRESS: at 40.04% examples, 312422 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:33,315 : INFO : PROGRESS: at 40.37% examples, 312396 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:34,325 : INFO : PROGRESS: at 40.70% examples, 312428 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:35,338 : INFO : PROGRESS: at 41.04% examples, 312343 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:36,359 : INFO : PROGRESS: at 41.36% examples, 312235 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:37,359 : INFO : PROGRESS: at 41.69% examples, 312187 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:38,372 : INFO : PROGRESS: at 42.00% examples, 312049 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:39,404 : INFO : PROGRESS: at 42.33% examples, 311931 words/s, in_qsize 5, out_qsize 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-12 16:11:40,406 : INFO : PROGRESS: at 42.67% examples, 311884 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:41,410 : INFO : PROGRESS: at 43.00% examples, 311833 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:42,455 : INFO : PROGRESS: at 43.34% examples, 311747 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:43,464 : INFO : PROGRESS: at 43.68% examples, 311687 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:44,478 : INFO : PROGRESS: at 44.00% examples, 311598 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:45,491 : INFO : PROGRESS: at 44.33% examples, 311584 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:46,496 : INFO : PROGRESS: at 44.65% examples, 311533 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:47,516 : INFO : PROGRESS: at 44.98% examples, 311501 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:48,516 : INFO : PROGRESS: at 45.31% examples, 311455 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:49,524 : INFO : PROGRESS: at 45.64% examples, 311393 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:11:50,542 : INFO : PROGRESS: at 45.97% examples, 311316 words/s, in_qsize 5, out_qsize 1\n",
      "2017-11-12 16:11:51,559 : INFO : PROGRESS: at 46.33% examples, 311294 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:52,566 : INFO : PROGRESS: at 46.65% examples, 311252 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:53,572 : INFO : PROGRESS: at 46.99% examples, 311251 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:54,573 : INFO : PROGRESS: at 47.30% examples, 311168 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:55,590 : INFO : PROGRESS: at 47.63% examples, 311148 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:56,624 : INFO : PROGRESS: at 47.97% examples, 311090 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:57,631 : INFO : PROGRESS: at 48.29% examples, 311093 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:58,633 : INFO : PROGRESS: at 48.59% examples, 311014 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:11:59,640 : INFO : PROGRESS: at 48.92% examples, 310967 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:00,652 : INFO : PROGRESS: at 49.25% examples, 310917 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:12:01,655 : INFO : PROGRESS: at 49.59% examples, 310921 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:12:02,689 : INFO : PROGRESS: at 49.94% examples, 310973 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:12:03,698 : INFO : PROGRESS: at 50.32% examples, 311252 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:04,732 : INFO : PROGRESS: at 50.69% examples, 311549 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:05,733 : INFO : PROGRESS: at 51.09% examples, 311855 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:06,749 : INFO : PROGRESS: at 51.46% examples, 312074 words/s, in_qsize 5, out_qsize 1\n",
      "2017-11-12 16:12:07,778 : INFO : PROGRESS: at 51.85% examples, 312318 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:08,782 : INFO : PROGRESS: at 52.17% examples, 312270 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:09,797 : INFO : PROGRESS: at 52.48% examples, 312180 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:10,798 : INFO : PROGRESS: at 52.81% examples, 312147 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:11,829 : INFO : PROGRESS: at 53.15% examples, 312095 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:12,832 : INFO : PROGRESS: at 53.48% examples, 312055 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:12:13,864 : INFO : PROGRESS: at 53.82% examples, 311997 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:12:14,892 : INFO : PROGRESS: at 54.16% examples, 311953 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:15,906 : INFO : PROGRESS: at 54.48% examples, 311938 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:16,920 : INFO : PROGRESS: at 54.81% examples, 311929 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:17,958 : INFO : PROGRESS: at 55.15% examples, 311866 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:18,983 : INFO : PROGRESS: at 55.48% examples, 311832 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:19,994 : INFO : PROGRESS: at 55.82% examples, 311821 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:21,001 : INFO : PROGRESS: at 56.14% examples, 311811 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:22,024 : INFO : PROGRESS: at 56.46% examples, 311743 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:23,038 : INFO : PROGRESS: at 56.82% examples, 311723 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:24,060 : INFO : PROGRESS: at 57.15% examples, 311656 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:25,073 : INFO : PROGRESS: at 57.48% examples, 311605 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:26,099 : INFO : PROGRESS: at 57.81% examples, 311570 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:27,122 : INFO : PROGRESS: at 58.13% examples, 311537 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:28,139 : INFO : PROGRESS: at 58.47% examples, 311473 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:29,155 : INFO : PROGRESS: at 58.81% examples, 311451 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:30,185 : INFO : PROGRESS: at 59.13% examples, 311415 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:31,190 : INFO : PROGRESS: at 59.46% examples, 311415 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:32,197 : INFO : PROGRESS: at 59.79% examples, 311382 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:33,198 : INFO : PROGRESS: at 59.95% examples, 310486 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:34,199 : INFO : PROGRESS: at 60.29% examples, 310456 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:35,214 : INFO : PROGRESS: at 60.64% examples, 310455 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:36,237 : INFO : PROGRESS: at 60.98% examples, 310429 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:37,254 : INFO : PROGRESS: at 61.32% examples, 310419 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:38,296 : INFO : PROGRESS: at 61.66% examples, 310397 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:39,303 : INFO : PROGRESS: at 61.99% examples, 310431 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:40,316 : INFO : PROGRESS: at 62.32% examples, 310392 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:41,366 : INFO : PROGRESS: at 62.66% examples, 310368 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:42,372 : INFO : PROGRESS: at 63.00% examples, 310369 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:43,377 : INFO : PROGRESS: at 63.33% examples, 310379 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:44,401 : INFO : PROGRESS: at 63.66% examples, 310320 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:45,403 : INFO : PROGRESS: at 63.99% examples, 310331 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:46,408 : INFO : PROGRESS: at 64.32% examples, 310332 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:47,412 : INFO : PROGRESS: at 64.64% examples, 310307 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:48,434 : INFO : PROGRESS: at 64.98% examples, 310323 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:49,438 : INFO : PROGRESS: at 65.31% examples, 310301 words/s, in_qsize 5, out_qsize 1\n",
      "2017-11-12 16:12:50,459 : INFO : PROGRESS: at 65.64% examples, 310321 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:12:51,510 : INFO : PROGRESS: at 65.96% examples, 310223 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:12:52,521 : INFO : PROGRESS: at 66.28% examples, 310189 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:53,548 : INFO : PROGRESS: at 66.61% examples, 310167 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:54,577 : INFO : PROGRESS: at 66.94% examples, 310110 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:55,642 : INFO : PROGRESS: at 67.27% examples, 310028 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:56,658 : INFO : PROGRESS: at 67.61% examples, 310021 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:57,665 : INFO : PROGRESS: at 67.92% examples, 309955 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:12:58,705 : INFO : PROGRESS: at 68.25% examples, 309874 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:12:59,717 : INFO : PROGRESS: at 68.58% examples, 309836 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:00,719 : INFO : PROGRESS: at 68.89% examples, 309752 words/s, in_qsize 5, out_qsize 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-12 16:13:01,723 : INFO : PROGRESS: at 69.20% examples, 309686 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:02,745 : INFO : PROGRESS: at 69.53% examples, 309709 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:03,763 : INFO : PROGRESS: at 69.88% examples, 309708 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:04,775 : INFO : PROGRESS: at 70.20% examples, 309705 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:05,776 : INFO : PROGRESS: at 70.54% examples, 309719 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:06,791 : INFO : PROGRESS: at 70.88% examples, 309711 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:07,803 : INFO : PROGRESS: at 71.21% examples, 309678 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:08,821 : INFO : PROGRESS: at 71.54% examples, 309701 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:09,829 : INFO : PROGRESS: at 71.87% examples, 309670 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:10,830 : INFO : PROGRESS: at 72.20% examples, 309652 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:11,847 : INFO : PROGRESS: at 72.52% examples, 309616 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:12,879 : INFO : PROGRESS: at 72.85% examples, 309618 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:13,902 : INFO : PROGRESS: at 73.19% examples, 309600 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:14,906 : INFO : PROGRESS: at 73.52% examples, 309614 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:15,930 : INFO : PROGRESS: at 73.86% examples, 309597 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:16,942 : INFO : PROGRESS: at 74.20% examples, 309572 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:17,975 : INFO : PROGRESS: at 74.54% examples, 309612 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:18,985 : INFO : PROGRESS: at 74.88% examples, 309619 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:20,001 : INFO : PROGRESS: at 75.23% examples, 309618 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:21,017 : INFO : PROGRESS: at 75.57% examples, 309619 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:22,046 : INFO : PROGRESS: at 75.90% examples, 309571 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:23,077 : INFO : PROGRESS: at 76.23% examples, 309547 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:24,078 : INFO : PROGRESS: at 76.56% examples, 309590 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:25,104 : INFO : PROGRESS: at 76.90% examples, 309567 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:26,139 : INFO : PROGRESS: at 77.25% examples, 309566 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:27,162 : INFO : PROGRESS: at 77.59% examples, 309554 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:28,168 : INFO : PROGRESS: at 77.92% examples, 309566 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:29,205 : INFO : PROGRESS: at 78.26% examples, 309563 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:30,241 : INFO : PROGRESS: at 78.61% examples, 309585 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:31,266 : INFO : PROGRESS: at 78.95% examples, 309574 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:32,309 : INFO : PROGRESS: at 79.29% examples, 309563 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:33,314 : INFO : PROGRESS: at 79.63% examples, 309603 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:34,560 : INFO : PROGRESS: at 79.92% examples, 309150 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:35,579 : INFO : PROGRESS: at 80.20% examples, 308936 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:36,614 : INFO : PROGRESS: at 80.53% examples, 308915 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:37,626 : INFO : PROGRESS: at 80.87% examples, 308922 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:38,627 : INFO : PROGRESS: at 81.20% examples, 308934 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:39,660 : INFO : PROGRESS: at 81.54% examples, 308946 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:40,687 : INFO : PROGRESS: at 81.89% examples, 308960 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:41,722 : INFO : PROGRESS: at 82.25% examples, 308939 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:42,724 : INFO : PROGRESS: at 82.59% examples, 308956 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:43,725 : INFO : PROGRESS: at 82.92% examples, 308938 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:44,751 : INFO : PROGRESS: at 83.26% examples, 308955 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:45,752 : INFO : PROGRESS: at 83.58% examples, 308971 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:46,774 : INFO : PROGRESS: at 83.92% examples, 308969 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:47,784 : INFO : PROGRESS: at 84.25% examples, 308921 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:48,818 : INFO : PROGRESS: at 84.60% examples, 308931 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:49,842 : INFO : PROGRESS: at 84.94% examples, 308922 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:50,849 : INFO : PROGRESS: at 85.26% examples, 308904 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:51,864 : INFO : PROGRESS: at 85.57% examples, 308896 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:52,886 : INFO : PROGRESS: at 85.92% examples, 308886 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:53,891 : INFO : PROGRESS: at 86.24% examples, 308899 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:13:54,919 : INFO : PROGRESS: at 86.57% examples, 308879 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:55,928 : INFO : PROGRESS: at 86.90% examples, 308859 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:56,956 : INFO : PROGRESS: at 87.24% examples, 308873 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:57,999 : INFO : PROGRESS: at 87.57% examples, 308844 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:13:59,037 : INFO : PROGRESS: at 87.91% examples, 308847 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:00,077 : INFO : PROGRESS: at 88.26% examples, 308819 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:01,093 : INFO : PROGRESS: at 88.61% examples, 308817 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:02,129 : INFO : PROGRESS: at 88.96% examples, 308823 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:14:03,169 : INFO : PROGRESS: at 89.29% examples, 308849 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:04,183 : INFO : PROGRESS: at 89.65% examples, 308879 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:05,225 : INFO : PROGRESS: at 89.98% examples, 308852 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:14:06,250 : INFO : PROGRESS: at 90.32% examples, 308845 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:07,301 : INFO : PROGRESS: at 90.66% examples, 308831 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:14:08,335 : INFO : PROGRESS: at 90.99% examples, 308811 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:09,355 : INFO : PROGRESS: at 91.31% examples, 308808 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:10,360 : INFO : PROGRESS: at 91.65% examples, 308819 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:14:11,376 : INFO : PROGRESS: at 91.96% examples, 308764 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:12,378 : INFO : PROGRESS: at 92.30% examples, 308779 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:14:13,405 : INFO : PROGRESS: at 92.64% examples, 308766 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:14,413 : INFO : PROGRESS: at 92.98% examples, 308747 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:15,443 : INFO : PROGRESS: at 93.30% examples, 308734 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:16,459 : INFO : PROGRESS: at 93.63% examples, 308730 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:17,514 : INFO : PROGRESS: at 93.98% examples, 308713 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:18,526 : INFO : PROGRESS: at 94.30% examples, 308721 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:19,532 : INFO : PROGRESS: at 94.64% examples, 308732 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:20,546 : INFO : PROGRESS: at 94.96% examples, 308711 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:21,568 : INFO : PROGRESS: at 95.29% examples, 308700 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:22,578 : INFO : PROGRESS: at 95.64% examples, 308705 words/s, in_qsize 5, out_qsize 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-12 16:14:23,596 : INFO : PROGRESS: at 95.97% examples, 308704 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:24,604 : INFO : PROGRESS: at 96.31% examples, 308716 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:25,605 : INFO : PROGRESS: at 96.63% examples, 308702 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:14:26,610 : INFO : PROGRESS: at 96.97% examples, 308714 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:27,629 : INFO : PROGRESS: at 97.30% examples, 308709 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:28,643 : INFO : PROGRESS: at 97.64% examples, 308710 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:29,644 : INFO : PROGRESS: at 97.96% examples, 308699 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:30,650 : INFO : PROGRESS: at 98.29% examples, 308711 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:31,660 : INFO : PROGRESS: at 98.62% examples, 308744 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:32,662 : INFO : PROGRESS: at 98.95% examples, 308753 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:33,686 : INFO : PROGRESS: at 99.29% examples, 308742 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:34,707 : INFO : PROGRESS: at 99.63% examples, 308733 words/s, in_qsize 6, out_qsize 0\n",
      "2017-11-12 16:14:35,711 : INFO : PROGRESS: at 99.96% examples, 308747 words/s, in_qsize 5, out_qsize 0\n",
      "2017-11-12 16:14:35,785 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-11-12 16:14:35,808 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-11-12 16:14:35,830 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-11-12 16:14:35,832 : INFO : training on 123467550 raw words (93770058 effective words) took 303.7s, 308747 effective words/s\n"
     ]
    }
   ],
   "source": [
    "permuter = PermuteSentences(unsup_sentences) \n",
    "model = Doc2Vec(permuter, dm=0, hs=1, size=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# done with training, free up some memory\n",
    "model.delete_temporary_training_data(keep_inference=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-12 16:14:35,953 : INFO : saving Doc2Vec object under reviews.d2v, separately None\n",
      "2017-11-12 16:14:35,956 : INFO : not storing attribute syn0norm\n",
      "2017-11-12 16:14:35,958 : INFO : not storing attribute cum_table\n",
      "2017-11-12 16:14:40,700 : INFO : saved reviews.d2v\n"
     ]
    }
   ],
   "source": [
    "model.save('reviews.d2v')\n",
    "# in other program, we could write: model = Doc2Vec.load('reviews.d2v')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.16600764,  0.29806057, -0.37614173,  0.58661956,  0.31548923,\n",
       "       -0.15109532, -0.19294184, -0.80975324, -0.13256417, -0.26431978,\n",
       "        0.15649557, -0.36540538, -0.33639464, -0.55479848, -0.02375498,\n",
       "        0.12179437, -0.06088163, -0.17349492, -0.19584687,  0.09399831,\n",
       "        0.01947556, -0.17546433, -0.07536539, -0.05634249,  0.2418247 ,\n",
       "       -0.11649339, -0.18398936, -0.37568066, -0.04755535, -0.23786636,\n",
       "        0.35202903, -0.25357839,  0.05126057, -0.22089498,  0.09130105,\n",
       "       -0.46730992,  0.34186646,  0.17174301,  0.51055247,  0.21438542,\n",
       "       -0.41699263, -0.5968706 , -0.00541743,  0.39446551,  0.07960459,\n",
       "       -0.20494871,  0.11499975,  0.22761559,  0.24039924, -0.06279976], dtype=float32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.infer_vector(extract_words(\"This place is not worth your time, let alone Vegas.\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.48211202]], dtype=float32)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "cosine_similarity(\n",
    "    [model.infer_vector(extract_words(\"This place is not worth your time, let alone Vegas.\"))],\n",
    "    [model.infer_vector(extract_words(\"Service sucks.\"))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.28899333]], dtype=float32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity(\n",
    "    [model.infer_vector(extract_words(\"Highly recommended.\"))],\n",
    "    [model.infer_vector(extract_words(\"Service sucks.\"))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sentences = []\n",
    "sentvecs = []\n",
    "sentiments = []\n",
    "for fname in [\"yelp\", \"amazon_cells\", \"imdb\"]: \n",
    "    with open(\"sentiment labelled sentences/%s_labelled.txt\" % fname, encoding='UTF-8') as f:\n",
    "        for i, line in enumerate(f):\n",
    "            line_split = line.strip().split('\\t')\n",
    "            sentences.append(line_split[0])\n",
    "            words = extract_words(line_split[0])\n",
    "            sentvecs.append(model.infer_vector(words, steps=10)) # create a vector for this document\n",
    "            sentiments.append(int(line_split[1]))\n",
    "            \n",
    "# shuffle sentences, sentvecs, sentiments together\n",
    "combined = list(zip(sentences, sentvecs, sentiments))\n",
    "random.shuffle(combined)\n",
    "sentences, sentvecs, sentiments = zip(*combined)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import numpy as np\n",
    "\n",
    "clf = KNeighborsClassifier(n_neighbors=9)\n",
    "clfrf = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.75900000000000012, 0.016950909513454807)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = cross_val_score(clf, sentvecs, sentiments, cv=5)\n",
    "np.mean(scores), np.std(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.69766666666666655, 0.019988885800753264)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = cross_val_score(clfrf, sentvecs, sentiments, cv=5)\n",
    "np.mean(scores), np.std(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# bag-of-words comparison\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "pipeline = make_pipeline(CountVectorizer(), TfidfTransformer(), RandomForestClassifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.73733333333333329, 0.015937377450509209)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = cross_val_score(pipeline, sentences, sentiments, cv=5)\n",
    "np.mean(scores), np.std(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
